{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:51.719433Z",
     "start_time": "2024-11-29T12:36:59.640734Z"
    }
   },
   "source": [
    "import pandas as pd\n",
    "\n",
    "\n",
    "\n",
    "paths = 'D:\\pythone-code\\pythonProject1\\\\tmfg\\data'\n",
    "\n",
    "data = pd.read_csv(f'{paths}/user_log_format1.csv', dtype={'time_stamp':'str'})\n",
    "\n",
    "data1 = pd.read_csv(f'{paths}/user_info_format1.csv')\n",
    "\n",
    "data2 = pd.read_csv(f'{paths}/train_format1.csv')\n",
    "\n",
    "submission = pd.read_csv(f'{paths}/test_format1.csv')\n",
    "\n",
    "data_train = pd.read_csv('D:\\pythone-code\\pythonProject1\\zg5\\\\tmfg\\data\\\\train_format2.csv')"
   ],
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mParserError\u001B[0m                               Traceback (most recent call last)",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\pandas\\io\\parsers\\readers.py:1748\u001B[0m, in \u001B[0;36mTextFileReader.read\u001B[1;34m(self, nrows)\u001B[0m\n\u001B[0;32m   1742\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m   1743\u001B[0m     \u001B[38;5;66;03m# error: \"ParserBase\" has no attribute \"read\"\u001B[39;00m\n\u001B[0;32m   1744\u001B[0m     (\n\u001B[0;32m   1745\u001B[0m         index,\n\u001B[0;32m   1746\u001B[0m         columns,\n\u001B[0;32m   1747\u001B[0m         col_dict,\n\u001B[1;32m-> 1748\u001B[0m     ) \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_engine\u001B[38;5;241m.\u001B[39mread(  \u001B[38;5;66;03m# type: ignore[attr-defined]\u001B[39;00m\n\u001B[0;32m   1749\u001B[0m         nrows\n\u001B[0;32m   1750\u001B[0m     )\n\u001B[0;32m   1751\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mException\u001B[39;00m:\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\pandas\\io\\parsers\\c_parser_wrapper.py:234\u001B[0m, in \u001B[0;36mCParserWrapper.read\u001B[1;34m(self, nrows)\u001B[0m\n\u001B[0;32m    233\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mlow_memory:\n\u001B[1;32m--> 234\u001B[0m     chunks \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_reader\u001B[38;5;241m.\u001B[39mread_low_memory(nrows)\n\u001B[0;32m    235\u001B[0m     \u001B[38;5;66;03m# destructive to chunks\u001B[39;00m\n",
      "File \u001B[1;32mparsers.pyx:843\u001B[0m, in \u001B[0;36mpandas._libs.parsers.TextReader.read_low_memory\u001B[1;34m()\u001B[0m\n",
      "File \u001B[1;32mparsers.pyx:904\u001B[0m, in \u001B[0;36mpandas._libs.parsers.TextReader._read_rows\u001B[1;34m()\u001B[0m\n",
      "File \u001B[1;32mparsers.pyx:879\u001B[0m, in \u001B[0;36mpandas._libs.parsers.TextReader._tokenize_rows\u001B[1;34m()\u001B[0m\n",
      "File \u001B[1;32mparsers.pyx:890\u001B[0m, in \u001B[0;36mpandas._libs.parsers.TextReader._check_tokenize_status\u001B[1;34m()\u001B[0m\n",
      "File \u001B[1;32mparsers.pyx:2058\u001B[0m, in \u001B[0;36mpandas._libs.parsers.raise_parser_error\u001B[1;34m()\u001B[0m\n",
      "\u001B[1;31mParserError\u001B[0m: Error tokenizing data. C error: Calling read(nbytes) on source failed. Try engine='python'.",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[37], line 15\u001B[0m\n\u001B[0;32m     11\u001B[0m data2 \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mread_csv(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mpaths\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m/train_format1.csv\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[0;32m     13\u001B[0m submission \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mread_csv(\u001B[38;5;124mf\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;132;01m{\u001B[39;00mpaths\u001B[38;5;132;01m}\u001B[39;00m\u001B[38;5;124m/test_format1.csv\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m---> 15\u001B[0m data_train \u001B[38;5;241m=\u001B[39m pd\u001B[38;5;241m.\u001B[39mread_csv(\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mD:\u001B[39m\u001B[38;5;124m\\\u001B[39m\u001B[38;5;124mpythone-code\u001B[39m\u001B[38;5;124m\\\u001B[39m\u001B[38;5;124mpythonProject1\u001B[39m\u001B[38;5;124m\\\u001B[39m\u001B[38;5;124mzg5\u001B[39m\u001B[38;5;130;01m\\\\\u001B[39;00m\u001B[38;5;124mtm\u001B[39m\u001B[38;5;124m\\\u001B[39m\u001B[38;5;124mdata1\u001B[39m\u001B[38;5;130;01m\\\\\u001B[39;00m\u001B[38;5;124mtrain_format2.csv\u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\pandas\\io\\parsers\\readers.py:948\u001B[0m, in \u001B[0;36mread_csv\u001B[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, date_format, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options, dtype_backend)\u001B[0m\n\u001B[0;32m    935\u001B[0m kwds_defaults \u001B[38;5;241m=\u001B[39m _refine_defaults_read(\n\u001B[0;32m    936\u001B[0m     dialect,\n\u001B[0;32m    937\u001B[0m     delimiter,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m    944\u001B[0m     dtype_backend\u001B[38;5;241m=\u001B[39mdtype_backend,\n\u001B[0;32m    945\u001B[0m )\n\u001B[0;32m    946\u001B[0m kwds\u001B[38;5;241m.\u001B[39mupdate(kwds_defaults)\n\u001B[1;32m--> 948\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m _read(filepath_or_buffer, kwds)\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\pandas\\io\\parsers\\readers.py:617\u001B[0m, in \u001B[0;36m_read\u001B[1;34m(filepath_or_buffer, kwds)\u001B[0m\n\u001B[0;32m    614\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m parser\n\u001B[0;32m    616\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m parser:\n\u001B[1;32m--> 617\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m parser\u001B[38;5;241m.\u001B[39mread(nrows)\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\pandas\\io\\parsers\\readers.py:1752\u001B[0m, in \u001B[0;36mTextFileReader.read\u001B[1;34m(self, nrows)\u001B[0m\n\u001B[0;32m   1744\u001B[0m     (\n\u001B[0;32m   1745\u001B[0m         index,\n\u001B[0;32m   1746\u001B[0m         columns,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m   1749\u001B[0m         nrows\n\u001B[0;32m   1750\u001B[0m     )\n\u001B[0;32m   1751\u001B[0m \u001B[38;5;28;01mexcept\u001B[39;00m \u001B[38;5;167;01mException\u001B[39;00m:\n\u001B[1;32m-> 1752\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mclose()\n\u001B[0;32m   1753\u001B[0m     \u001B[38;5;28;01mraise\u001B[39;00m\n\u001B[0;32m   1755\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m index \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\pandas\\io\\parsers\\readers.py:1452\u001B[0m, in \u001B[0;36mTextFileReader.close\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m   1450\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mclose\u001B[39m(\u001B[38;5;28mself\u001B[39m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m   1451\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mhandles \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m-> 1452\u001B[0m         \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mhandles\u001B[38;5;241m.\u001B[39mclose()\n\u001B[0;32m   1453\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_engine\u001B[38;5;241m.\u001B[39mclose()\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\pandas\\io\\common.py:121\u001B[0m, in \u001B[0;36mIOHandles.close\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    118\u001B[0m created_handles: \u001B[38;5;28mlist\u001B[39m[IO[\u001B[38;5;28mbytes\u001B[39m] \u001B[38;5;241m|\u001B[39m IO[\u001B[38;5;28mstr\u001B[39m]] \u001B[38;5;241m=\u001B[39m dataclasses\u001B[38;5;241m.\u001B[39mfield(default_factory\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mlist\u001B[39m)\n\u001B[0;32m    119\u001B[0m is_wrapped: \u001B[38;5;28mbool\u001B[39m \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[1;32m--> 121\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mclose\u001B[39m(\u001B[38;5;28mself\u001B[39m) \u001B[38;5;241m-\u001B[39m\u001B[38;5;241m>\u001B[39m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[0;32m    122\u001B[0m \u001B[38;5;250m    \u001B[39m\u001B[38;5;124;03m\"\"\"\u001B[39;00m\n\u001B[0;32m    123\u001B[0m \u001B[38;5;124;03m    Close all created buffers.\u001B[39;00m\n\u001B[0;32m    124\u001B[0m \n\u001B[0;32m    125\u001B[0m \u001B[38;5;124;03m    Note: If a TextIOWrapper was inserted, it is flushed and detached to\u001B[39;00m\n\u001B[0;32m    126\u001B[0m \u001B[38;5;124;03m    avoid closing the potentially user-created buffer.\u001B[39;00m\n\u001B[0;32m    127\u001B[0m \u001B[38;5;124;03m    \"\"\"\u001B[39;00m\n\u001B[0;32m    128\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mis_wrapped:\n",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "execution_count": 37
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.073599Z",
     "start_time": "2024-11-29T12:01:14.586321Z"
    }
   },
   "cell_type": "code",
   "source": [
    "data2['origin'] = 'train'\n",
    "\n",
    "submission['origin'] = 'test'\n",
    "\n",
    "matrix = pd.concat([data2, submission], ignore_index=True, sort=False)\n",
    "\n",
    "matrix.drop(['prob'], axis=1, inplace=True)\n",
    "\n",
    "matrix = matrix.merge(data1, on='user_id', how='left')\n",
    "\n",
    "data.rename(columns={'seller_id':'merchant_id'}, inplace=True)"
   ],
   "id": "496568a5aeccc921",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.073599Z",
     "start_time": "2024-11-29T12:01:15.186Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import gc\n",
    "\n",
    "data['user_id'] = data['user_id'].astype('int32')\n",
    "\n",
    "data['merchant_id'] = data['merchant_id'].astype('int32')\n",
    "\n",
    "data['item_id'] = data['item_id'].astype('int32')\n",
    "\n",
    "data['cat_id'] = data['cat_id'].astype('int32')\n",
    "\n",
    "data['brand_id'].fillna(0, inplace=True)\n",
    "\n",
    "data['brand_id'] = data['brand_id'].astype('int32')\n",
    "\n",
    "data['time_stamp'] = pd.to_datetime(data['time_stamp'], format='%H%M')\n",
    "\n",
    "matrix['age_range'].fillna(0, inplace=True)\n",
    "\n",
    "matrix['gender'].fillna(2, inplace=True)\n",
    "\n",
    "matrix['age_range'] = matrix['age_range'].astype('int8')\n",
    "\n",
    "matrix['gender'] = matrix['gender'].astype('int8')\n",
    "\n",
    "matrix['label'] = matrix['label'].astype('str')\n",
    "\n",
    "matrix['user_id'] = matrix['user_id'].astype('int32')\n",
    "\n",
    "matrix['merchant_id'] = matrix['merchant_id'].astype('int32')\n",
    "\n",
    "del data1, data2\n",
    "\n",
    "gc.collect()"
   ],
   "id": "1cd4246e767e28d5",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "74"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.073599Z",
     "start_time": "2024-11-29T12:01:22.347278Z"
    }
   },
   "cell_type": "code",
   "source": [
    "groups = data.groupby(['user_id'])\n",
    "\n",
    "temp = groups.size().reset_index().rename(columns={0:'u1'})\n",
    "\n",
    "matrix = matrix.merge(temp, on='user_id', how='left')\n",
    "\n",
    "temp = groups['item_id'].agg([('u2', 'nunique')]).reset_index()\n",
    "\n",
    "matrix = matrix.merge(temp, on='user_id', how='left')\n",
    "\n",
    "temp = groups['cat_id'].agg([('u3', 'nunique')]).reset_index()\n",
    "\n",
    "matrix = matrix.merge(temp, on='user_id', how='left')\n",
    "\n",
    "temp = groups['merchant_id'].agg([('u4', 'nunique')]).reset_index()\n",
    "\n",
    "matrix = matrix.merge(temp, on='user_id', how='left')\n",
    "\n",
    "temp = groups['brand_id'].agg([('u5', 'nunique')]).reset_index()\n",
    "\n",
    "matrix = matrix.merge(temp, on='user_id', how='left')\n",
    "\n",
    "temp = groups['time_stamp'].agg([('F_time', 'min'), ('L_time', 'max')]).reset_index()\n",
    "\n",
    "temp['u6'] = (temp['L_time'] - temp['F_time']).dt.seconds/3600\n",
    "\n",
    "matrix = matrix.merge(temp[['user_id', 'u6']], on='user_id', how='left')\n",
    "\n",
    "temp = groups['action_type'].value_counts().unstack().reset_index().rename(columns={0:'u7', 1:'u8', 2:'u9', 3:'u10'})\n",
    "\n",
    "matrix = matrix.merge(temp, on='user_id', how='left')"
   ],
   "id": "e0b5ecf07869222",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.074596800Z",
     "start_time": "2024-11-29T12:05:03.461499Z"
    }
   },
   "cell_type": "code",
   "source": [
    "groups = data.groupby(['merchant_id'])\n",
    "\n",
    "temp = groups.size().reset_index().rename(columns={0:'m1'})\n",
    "\n",
    "matrix = matrix.merge(temp, on='merchant_id', how='left')\n",
    "\n",
    "temp = groups[['user_id', 'item_id', 'cat_id', 'brand_id']].nunique().reset_index().rename(columns={\n",
    "    'user_id':'m2',\n",
    "    'item_id':'m3', \n",
    "    'cat_id':'m4', \n",
    "    'brand_id':'m5'})\n",
    "matrix = matrix.merge(temp, on='merchant_id', how='left')\n",
    "\n",
    "temp = groups['action_type'].value_counts().unstack().reset_index().rename(columns={0:'m6', 1:'m7', 2:'m8', 3:'m9'})\n",
    "\n",
    "matrix = matrix.merge(temp, on='merchant_id', how='left')"
   ],
   "id": "e4c81a1b993b4e6b",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.074596800Z",
     "start_time": "2024-11-29T12:36:53.680124Z"
    }
   },
   "cell_type": "code",
   "source": [
    "temp = data_train[data_train['label']==1].groupby(['merchant_id']).size().reset_index().rename(columns={0:'m10'})\n",
    "\n",
    "matrix = matrix.merge(temp, on='merchant_id', how='left')"
   ],
   "id": "98817e278c5d415",
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'matrix' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[36], line 3\u001B[0m\n\u001B[0;32m      1\u001B[0m temp \u001B[38;5;241m=\u001B[39m data_train[data_train[\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mlabel\u001B[39m\u001B[38;5;124m'\u001B[39m]\u001B[38;5;241m==\u001B[39m\u001B[38;5;241m1\u001B[39m]\u001B[38;5;241m.\u001B[39mgroupby([\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mmerchant_id\u001B[39m\u001B[38;5;124m'\u001B[39m])\u001B[38;5;241m.\u001B[39msize()\u001B[38;5;241m.\u001B[39mreset_index()\u001B[38;5;241m.\u001B[39mrename(columns\u001B[38;5;241m=\u001B[39m{\u001B[38;5;241m0\u001B[39m:\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mm10\u001B[39m\u001B[38;5;124m'\u001B[39m})\n\u001B[1;32m----> 3\u001B[0m matrix \u001B[38;5;241m=\u001B[39m matrix\u001B[38;5;241m.\u001B[39mmerge(temp, on\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mmerchant_id\u001B[39m\u001B[38;5;124m'\u001B[39m, how\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mleft\u001B[39m\u001B[38;5;124m'\u001B[39m)\n",
      "\u001B[1;31mNameError\u001B[0m: name 'matrix' is not defined"
     ]
    }
   ],
   "execution_count": 36
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.074596800Z",
     "start_time": "2024-11-29T12:08:53.160626Z"
    }
   },
   "cell_type": "code",
   "source": [
    "groups = data.groupby(['user_id', 'merchant_id'])\n",
    "\n",
    "temp = groups.size().reset_index().rename(columns={0:'um1'})\n",
    "\n",
    "matrix = matrix.merge(temp, on=['user_id', 'merchant_id'], how='left')\n",
    "\n",
    "temp = groups[['item_id', 'cat_id', 'brand_id']].nunique().reset_index().rename(columns={\n",
    "\n",
    "    'item_id':'um2',\n",
    "\n",
    "    'cat_id':'um3',\n",
    "\n",
    "    'brand_id':'um4'\n",
    "\n",
    "})\n",
    "\n",
    "matrix = matrix.merge(temp, on=['user_id', 'merchant_id'], how='left')\n",
    "\n",
    "temp = groups['action_type'].value_counts().unstack().reset_index().rename(columns={\n",
    "\n",
    "    0:'um5',\n",
    "\n",
    "    1:'um6',\n",
    "\n",
    "    2:'um7',\n",
    "\n",
    "    3:'um8'\n",
    "\n",
    "})\n",
    "\n",
    "matrix = matrix.merge(temp, on=['user_id', 'merchant_id'], how='left')\n",
    "\n",
    "temp = groups['time_stamp'].agg([('frist', 'min'), ('last', 'max')]).reset_index()\n",
    "\n",
    "temp['um9'] = (temp['last'] - temp['frist']).dt.seconds/3600\n",
    "\n",
    "temp.drop(['frist', 'last'], axis=1, inplace=True)\n",
    "\n",
    "matrix = matrix.merge(temp, on=['user_id', 'merchant_id'], how='left')"
   ],
   "id": "c22a19372c8668e5",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.074596800Z",
     "start_time": "2024-11-29T12:16:55.092684Z"
    }
   },
   "cell_type": "code",
   "source": [
    "matrix['r1'] = matrix['u9']/matrix['u7'] #用户购买点击比\n",
    "\n",
    "matrix['r2'] = matrix['m8']/matrix['m6'] #商家购买点击比\n",
    "\n",
    "matrix['r3'] = matrix['um7']/matrix['um5'] #不同用户不同商家购买点击比"
   ],
   "id": "7b1893017e93521f",
   "outputs": [],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.074596800Z",
     "start_time": "2024-11-29T12:16:55.110887Z"
    }
   },
   "cell_type": "code",
   "source": "matrix.fillna(0, inplace=True)",
   "id": "5cfbb8eb02b34d13",
   "outputs": [],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.074596800Z",
     "start_time": "2024-11-29T12:16:55.332262Z"
    }
   },
   "cell_type": "code",
   "source": [
    "temp = pd.get_dummies(matrix['age_range'], prefix='age')\n",
    "\n",
    "matrix = pd.concat([matrix, temp], axis=1)\n",
    "\n",
    "temp = pd.get_dummies(matrix['gender'], prefix='g')\n",
    "\n",
    "matrix = pd.concat([matrix, temp], axis=1)\n",
    "\n",
    "matrix.drop(['age_range', 'gender'], axis=1, inplace=True)"
   ],
   "id": "afa3fd189fe1289e",
   "outputs": [],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.074596800Z",
     "start_time": "2024-11-29T12:16:56.016882Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#train、test-setdata\n",
    "\n",
    "train_data = matrix[matrix['origin'] == 'train'].drop(['origin'], axis=1)\n",
    "\n",
    "test_data = matrix[matrix['origin'] == 'test'].drop(['label', 'origin'], axis=1)\n",
    "\n",
    "train_X, train_y = train_data.drop(['label'], axis=1), train_data['label']\n",
    "\n",
    "\n",
    "del temp, matrix\n",
    "\n",
    "gc.collect()"
   ],
   "id": "a1c9ca68e697a41c",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.074596800Z",
     "start_time": "2024-11-29T12:17:03.747416Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#导入分析库\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "import xgboost as xgb"
   ],
   "id": "9cb8f692a83ab01d",
   "outputs": [],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.075593Z",
     "start_time": "2024-11-29T12:17:08.744291Z"
    }
   },
   "cell_type": "code",
   "source": "X_train, X_valid, y_train, y_valid = train_test_split(train_X, train_y, test_size=.3)\n",
   "id": "63834e9c2d786aa9",
   "outputs": [],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T13:20:28.108629Z",
     "start_time": "2024-11-29T13:20:23.794018Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "\n",
    "rf_clf = RandomForestClassifier(\n",
    "\n",
    "    oob_score=True, \n",
    "\n",
    "    n_jobs=-1, \n",
    "\n",
    "    n_estimators=1000, \n",
    "\n",
    "    max_depth=10, \n",
    "\n",
    "    max_features='sqrt')\n",
    "\n",
    "\n",
    "rf_clf.fit(X_train, y_train)\n",
    "\n"
   ],
   "id": "a87b3c57d5e46cb4",
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m                         Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[38], line 14\u001B[0m\n\u001B[0;32m      1\u001B[0m rf_clf \u001B[38;5;241m=\u001B[39m RandomForestClassifier(\n\u001B[0;32m      2\u001B[0m \n\u001B[0;32m      3\u001B[0m     oob_score\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m, \n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m     10\u001B[0m \n\u001B[0;32m     11\u001B[0m     max_features\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124msqrt\u001B[39m\u001B[38;5;124m'\u001B[39m)\n\u001B[1;32m---> 14\u001B[0m rf_clf\u001B[38;5;241m.\u001B[39mfit(X_train, y_train)\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\sklearn\\ensemble\\_forest.py:473\u001B[0m, in \u001B[0;36mBaseForest.fit\u001B[1;34m(self, X, y, sample_weight)\u001B[0m\n\u001B[0;32m    462\u001B[0m trees \u001B[38;5;241m=\u001B[39m [\n\u001B[0;32m    463\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_make_estimator(append\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mFalse\u001B[39;00m, random_state\u001B[38;5;241m=\u001B[39mrandom_state)\n\u001B[0;32m    464\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m i \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mrange\u001B[39m(n_more_estimators)\n\u001B[0;32m    465\u001B[0m ]\n\u001B[0;32m    467\u001B[0m \u001B[38;5;66;03m# Parallel loop: we prefer the threading backend as the Cython code\u001B[39;00m\n\u001B[0;32m    468\u001B[0m \u001B[38;5;66;03m# for fitting the trees is internally releasing the Python GIL\u001B[39;00m\n\u001B[0;32m    469\u001B[0m \u001B[38;5;66;03m# making threading more efficient than multiprocessing in\u001B[39;00m\n\u001B[0;32m    470\u001B[0m \u001B[38;5;66;03m# that case. However, for joblib 0.12+ we respect any\u001B[39;00m\n\u001B[0;32m    471\u001B[0m \u001B[38;5;66;03m# parallel_backend contexts set at a higher level,\u001B[39;00m\n\u001B[0;32m    472\u001B[0m \u001B[38;5;66;03m# since correctness does not rely on using threads.\u001B[39;00m\n\u001B[1;32m--> 473\u001B[0m trees \u001B[38;5;241m=\u001B[39m Parallel(\n\u001B[0;32m    474\u001B[0m     n_jobs\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mn_jobs,\n\u001B[0;32m    475\u001B[0m     verbose\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose,\n\u001B[0;32m    476\u001B[0m     prefer\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124mthreads\u001B[39m\u001B[38;5;124m\"\u001B[39m,\n\u001B[0;32m    477\u001B[0m )(\n\u001B[0;32m    478\u001B[0m     delayed(_parallel_build_trees)(\n\u001B[0;32m    479\u001B[0m         t,\n\u001B[0;32m    480\u001B[0m         \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mbootstrap,\n\u001B[0;32m    481\u001B[0m         X,\n\u001B[0;32m    482\u001B[0m         y,\n\u001B[0;32m    483\u001B[0m         sample_weight,\n\u001B[0;32m    484\u001B[0m         i,\n\u001B[0;32m    485\u001B[0m         \u001B[38;5;28mlen\u001B[39m(trees),\n\u001B[0;32m    486\u001B[0m         verbose\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mverbose,\n\u001B[0;32m    487\u001B[0m         class_weight\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mclass_weight,\n\u001B[0;32m    488\u001B[0m         n_samples_bootstrap\u001B[38;5;241m=\u001B[39mn_samples_bootstrap,\n\u001B[0;32m    489\u001B[0m     )\n\u001B[0;32m    490\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m i, t \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28menumerate\u001B[39m(trees)\n\u001B[0;32m    491\u001B[0m )\n\u001B[0;32m    493\u001B[0m \u001B[38;5;66;03m# Collect newly grown trees\u001B[39;00m\n\u001B[0;32m    494\u001B[0m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mestimators_\u001B[38;5;241m.\u001B[39mextend(trees)\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\sklearn\\utils\\parallel.py:63\u001B[0m, in \u001B[0;36mParallel.__call__\u001B[1;34m(self, iterable)\u001B[0m\n\u001B[0;32m     58\u001B[0m config \u001B[38;5;241m=\u001B[39m get_config()\n\u001B[0;32m     59\u001B[0m iterable_with_config \u001B[38;5;241m=\u001B[39m (\n\u001B[0;32m     60\u001B[0m     (_with_config(delayed_func, config), args, kwargs)\n\u001B[0;32m     61\u001B[0m     \u001B[38;5;28;01mfor\u001B[39;00m delayed_func, args, kwargs \u001B[38;5;129;01min\u001B[39;00m iterable\n\u001B[0;32m     62\u001B[0m )\n\u001B[1;32m---> 63\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28msuper\u001B[39m()\u001B[38;5;241m.\u001B[39m\u001B[38;5;21m__call__\u001B[39m(iterable_with_config)\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\joblib\\parallel.py:1098\u001B[0m, in \u001B[0;36mParallel.__call__\u001B[1;34m(self, iterable)\u001B[0m\n\u001B[0;32m   1095\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_iterating \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mFalse\u001B[39;00m\n\u001B[0;32m   1097\u001B[0m \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backend\u001B[38;5;241m.\u001B[39mretrieval_context():\n\u001B[1;32m-> 1098\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mretrieve()\n\u001B[0;32m   1099\u001B[0m \u001B[38;5;66;03m# Make sure that we get a last message telling us we are done\u001B[39;00m\n\u001B[0;32m   1100\u001B[0m elapsed_time \u001B[38;5;241m=\u001B[39m time\u001B[38;5;241m.\u001B[39mtime() \u001B[38;5;241m-\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_start_time\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\joblib\\parallel.py:975\u001B[0m, in \u001B[0;36mParallel.retrieve\u001B[1;34m(self)\u001B[0m\n\u001B[0;32m    973\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:\n\u001B[0;32m    974\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;28mgetattr\u001B[39m(\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_backend, \u001B[38;5;124m'\u001B[39m\u001B[38;5;124msupports_timeout\u001B[39m\u001B[38;5;124m'\u001B[39m, \u001B[38;5;28;01mFalse\u001B[39;00m):\n\u001B[1;32m--> 975\u001B[0m         \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_output\u001B[38;5;241m.\u001B[39mextend(job\u001B[38;5;241m.\u001B[39mget(timeout\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mtimeout))\n\u001B[0;32m    976\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m    977\u001B[0m         \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_output\u001B[38;5;241m.\u001B[39mextend(job\u001B[38;5;241m.\u001B[39mget())\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\multiprocessing\\pool.py:768\u001B[0m, in \u001B[0;36mApplyResult.get\u001B[1;34m(self, timeout)\u001B[0m\n\u001B[0;32m    767\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mget\u001B[39m(\u001B[38;5;28mself\u001B[39m, timeout\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[1;32m--> 768\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mwait(timeout)\n\u001B[0;32m    769\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mready():\n\u001B[0;32m    770\u001B[0m         \u001B[38;5;28;01mraise\u001B[39;00m \u001B[38;5;167;01mTimeoutError\u001B[39;00m\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\multiprocessing\\pool.py:765\u001B[0m, in \u001B[0;36mApplyResult.wait\u001B[1;34m(self, timeout)\u001B[0m\n\u001B[0;32m    764\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m \u001B[38;5;21mwait\u001B[39m(\u001B[38;5;28mself\u001B[39m, timeout\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mNone\u001B[39;00m):\n\u001B[1;32m--> 765\u001B[0m     \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_event\u001B[38;5;241m.\u001B[39mwait(timeout)\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\threading.py:629\u001B[0m, in \u001B[0;36mEvent.wait\u001B[1;34m(self, timeout)\u001B[0m\n\u001B[0;32m    627\u001B[0m signaled \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_flag\n\u001B[0;32m    628\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m signaled:\n\u001B[1;32m--> 629\u001B[0m     signaled \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_cond\u001B[38;5;241m.\u001B[39mwait(timeout)\n\u001B[0;32m    630\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m signaled\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\threading.py:327\u001B[0m, in \u001B[0;36mCondition.wait\u001B[1;34m(self, timeout)\u001B[0m\n\u001B[0;32m    325\u001B[0m \u001B[38;5;28;01mtry\u001B[39;00m:    \u001B[38;5;66;03m# restore state no matter what (e.g., KeyboardInterrupt)\u001B[39;00m\n\u001B[0;32m    326\u001B[0m     \u001B[38;5;28;01mif\u001B[39;00m timeout \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n\u001B[1;32m--> 327\u001B[0m         waiter\u001B[38;5;241m.\u001B[39macquire()\n\u001B[0;32m    328\u001B[0m         gotit \u001B[38;5;241m=\u001B[39m \u001B[38;5;28;01mTrue\u001B[39;00m\n\u001B[0;32m    329\u001B[0m     \u001B[38;5;28;01melse\u001B[39;00m:\n",
      "\u001B[1;31mKeyboardInterrupt\u001B[0m: "
     ]
    }
   ],
   "execution_count": 38
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.076602200Z",
     "start_time": "2024-11-29T12:28:35.500550Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "y_pred_proba = rf_clf.predict_proba(X_valid)[:,1]\n",
    "auc = roc_auc_score(y_valid, y_pred_proba)\n",
    "auc"
   ],
   "id": "2055aa36b43a6575",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6592538814786146"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 30
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T13:20:31.118689Z",
     "start_time": "2024-11-29T13:20:31.062548Z"
    }
   },
   "cell_type": "code",
   "source": [
    "model = xgb.XGBClassifier(\n",
    "    max_depth=8,\n",
    "    n_estimators=1000,\n",
    "    min_child_weight=300,\n",
    "    colsample_bytree=0.8,\n",
    "    subsample=0.8,\n",
    "    eta=0.3,\n",
    "    seed=42,\n",
    "    eval_metric='auc'  # 正确位置设置eval_metric参数\n",
    ")\n",
    "\n",
    "model.fit(\n",
    "    X_train,\n",
    "    y_train,\n",
    "    eval_set=[(X_train, y_train), (X_valid, y_valid)],\n",
    "    verbose=True,\n",
    "    early_stopping_rounds=10\n",
    ")"
   ],
   "id": "7d101edddf438eec",
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "XGBClassifier.fit() got an unexpected keyword argument 'early_stopping_rounds'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mTypeError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[39], line 12\u001B[0m\n\u001B[0;32m      1\u001B[0m model \u001B[38;5;241m=\u001B[39m xgb\u001B[38;5;241m.\u001B[39mXGBClassifier(\n\u001B[0;32m      2\u001B[0m     max_depth\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m8\u001B[39m,\n\u001B[0;32m      3\u001B[0m     n_estimators\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m1000\u001B[39m,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m      9\u001B[0m     eval_metric\u001B[38;5;241m=\u001B[39m\u001B[38;5;124m'\u001B[39m\u001B[38;5;124mauc\u001B[39m\u001B[38;5;124m'\u001B[39m  \u001B[38;5;66;03m# 正确位置设置eval_metric参数\u001B[39;00m\n\u001B[0;32m     10\u001B[0m )\n\u001B[1;32m---> 12\u001B[0m model\u001B[38;5;241m.\u001B[39mfit(\n\u001B[0;32m     13\u001B[0m     X_train,\n\u001B[0;32m     14\u001B[0m     y_train,\n\u001B[0;32m     15\u001B[0m     eval_set\u001B[38;5;241m=\u001B[39m[(X_train, y_train), (X_valid, y_valid)],\n\u001B[0;32m     16\u001B[0m     verbose\u001B[38;5;241m=\u001B[39m\u001B[38;5;28;01mTrue\u001B[39;00m,\n\u001B[0;32m     17\u001B[0m     early_stopping_rounds\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m10\u001B[39m\n\u001B[0;32m     18\u001B[0m )\n",
      "File \u001B[1;32mD:\\annocode\\Lib\\site-packages\\xgboost\\core.py:726\u001B[0m, in \u001B[0;36mrequire_keyword_args.<locals>.throw_if.<locals>.inner_f\u001B[1;34m(*args, **kwargs)\u001B[0m\n\u001B[0;32m    724\u001B[0m \u001B[38;5;28;01mfor\u001B[39;00m k, arg \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mzip\u001B[39m(sig\u001B[38;5;241m.\u001B[39mparameters, args):\n\u001B[0;32m    725\u001B[0m     kwargs[k] \u001B[38;5;241m=\u001B[39m arg\n\u001B[1;32m--> 726\u001B[0m \u001B[38;5;28;01mreturn\u001B[39;00m func(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
      "\u001B[1;31mTypeError\u001B[0m: XGBClassifier.fit() got an unexpected keyword argument 'early_stopping_rounds'"
     ]
    }
   ],
   "execution_count": 39
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-29T12:38:52.076602200Z",
     "start_time": "2024-11-29T12:25:54.333599Z"
    }
   },
   "cell_type": "code",
   "source": [
    "prob = rf_clf.predict_proba(test_data)\n",
    "\n",
    "submission['prob'] = pd.Series(prob[:,1])\n",
    "\n",
    "submission.drop(['origin'], axis=1, inplace=True)\n",
    "\n",
    "submission.to_csv('submission.csv', index=False)"
   ],
   "id": "7292e81ac47a6782",
   "outputs": [],
   "execution_count": 27
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
